{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5d244f26",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.9998288 , -0.01850212, -0.4979533 ], dtype=float32)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gym\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "\n",
    "    def __init__(self):\n",
    "        env = gym.make('Pendulum-v1', render_mode='rgb_array')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state, _ = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, done, _, info = self.env.step(action)\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 200:\n",
    "            done = True\n",
    "        return state, reward, done, info\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b77c66d3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAR8AAAEXCAYAAACUBEAgAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAarUlEQVR4nO3dfVBU99028GsX2AWEswjKrowQuUcTw/jSBhW3+SPz1K3EkDRG8kzqOIaxPmY0q6OS8a60SqaZzuCjM01NazQznar3H4YOmWKq1aQ8aLAZV1SUBt9oOrcGbnWXqGUXUBbY/T5/GE5dJZblZX+79PrMnBn3/H7LXoewV87LvhhEREBEFGFG1QGI6N8Ty4eIlGD5EJESLB8iUoLlQ0RKsHyISAmWDxEpwfIhIiVYPkSkBMuHiJRQVj67du3ClClTkJiYiIKCApw+fVpVFCJSQEn5/P73v0dpaSnefvttnDt3DrNnz0ZhYSHa2tpUxCEiBQwq3lhaUFCAuXPn4je/+Q0AIBgMIjs7G+vWrcPmzZv/5f2DwSBu3LiB1NRUGAyG0Y5LRIMkIujo6EBWVhaMxsfv28RHKJOup6cHDQ0NKCsr09cZjUY4HA64XK4B7+P3++H3+/Xb169fR15e3qhnJaKhaW1txeTJkx87J+Llc+vWLQQCAVit1pD1VqsVV65cGfA+FRUV+PnPf/7I+tbWVmiaNio5iSh8Pp8P2dnZSE1N/ZdzI14+Q1FWVobS0lL9dv8GaprG8iGKQoM5HRLx8pkwYQLi4uLg8XhC1ns8HthstgHvYzabYTabIxGPiCIk4le7TCYT8vPzUVtbq68LBoOora2F3W6PdBwiUkTJYVdpaSlKSkowZ84czJs3D7/61a/Q1dWFFStWqIhDRAooKZ/XXnsNX3/9NcrLy+F2u/Gd73wHn3zyySMnoYlo7FLyOp/h8vl8sFgs8Hq9POFMFEXCeW7yvV1EpATLh4iUYPkQkRIsHyJSguVDREqwfIhICZYPESnB8iEiJVg+RKQEy4eIlGD5EJESLB8iUoLlQ0RKsHyISAmWDxEpwfIhIiVYPkSkBMuHiJRg+RCREiwfIlKC5UNESrB8iEgJlg8RKcHyISIlWD5EpATLh4iUYPkQkRIsHyJSguVDREqwfIhICZYPESnB8iEiJVg+RKQEy4eIlGD5EJESYZfPiRMn8NJLLyErKwsGgwEHDx4MGRcRlJeXY9KkSUhKSoLD4cCXX34ZMufOnTtYtmwZNE1DWloaVq5cic7OzmFtCBHFlrDLp6urC7Nnz8auXbsGHN++fTvee+897NmzB/X19Rg3bhwKCwvR3d2tz1m2bBkuXryImpoaHD58GCdOnMAbb7wx9K0gotgjwwBAqqur9dvBYFBsNpvs2LFDX9fe3i5ms1k+/PBDERG5dOmSAJAzZ87oc44ePSoGg0GuX78+qMf1er0CQLxe73DiE9EIC+e5OaLnfK5evQq32w2Hw6Gvs1gsKCgogMvlAgC4XC6kpaVhzpw5+hyHwwGj0Yj6+voBf67f74fP5wtZiCi2jWj5uN1uAIDVag1Zb7Va9TG3243MzMyQ8fj4eKSnp+tzHlZRUQGLxaIv2dnZIxmbiBSIiatdZWVl8Hq9+tLa2qo6EhEN04iWj81mAwB4PJ6Q9R6PRx+z2Wxoa2sLGe/r68OdO3f0OQ8zm83QNC1kIaLYNqLlk5ubC5vNhtraWn2dz+dDfX097HY7AMBut6O9vR0NDQ36nGPHjiEYDKKgoGAk4xBRFIsP9w6dnZ34+9//rt++evUqGhsbkZ6ejpycHGzYsAG/+MUvMG3aNOTm5mLr1q3IysrC4sWLAQBPP/00nn/+eaxatQp79uxBb28v1q5dix/96EfIysoasQ0joigX7qW048ePC4BHlpKSEhG5f7l969atYrVaxWw2y4IFC6S5uTnkZ9y+fVuWLl0qKSkpommarFixQjo6OgadgZfaiaJTOM9Ng4iIwu4bEp/PB4vFAq/Xy/M/RFEknOdmTFztIqKxh+VDREqwfIhICZYPESnB8iEiJVg+RKQEy4eIlGD5EJESLB8iUoLlQ0RKsHyISAmWDxEpwfIhIiVYPkSkBMuHiJRg+RCREiwfIlKC5UNESrB8iEgJlg8RKcHyISIlWD5EpATLh4iUYPkQkRIsHyJSguVDREqwfIhICZYPESnB8iEiJVg+RKQEy4eIlGD5EJESLB8iUoLlQ0RKsHyISImwyqeiogJz585FamoqMjMzsXjxYjQ3N4fM6e7uhtPpREZGBlJSUlBcXAyPxxMyp6WlBUVFRUhOTkZmZiY2bdqEvr6+4W8NEcWMsMqnrq4OTqcTp06dQk1NDXp7e7Fw4UJ0dXXpczZu3IhDhw6hqqoKdXV1uHHjBpYsWaKPBwIBFBUVoaenBydPnsT+/fuxb98+lJeXj9xWEVH0k2Foa2sTAFJXVyciIu3t7ZKQkCBVVVX6nMuXLwsAcblcIiJy5MgRMRqN4na79Tm7d+8WTdPE7/cP6nG9Xq8AEK/XO5z4RDTCwnluDuucj9frBQCkp6cDABoaGtDb2wuHw6HPmT59OnJycuByuQAALpcLM2fOhNVq1ecUFhbC5/Ph4sWLAz6O3++Hz+cLWYgotg25fILBIDZs2IBnn30WM2bMAAC43W6YTCakpaWFzLVarXC73fqcB4unf7x/bCAVFRWwWCz6kp2dPdTYRBQlhlw+TqcTFy5cQGVl5UjmGVBZWRm8Xq++tLa2jvpjEtHoih/KndauXYvDhw/jxIkTmDx5sr7eZrOhp6cH7e3tIXs/Ho8HNptNn3P69OmQn9d/Nax/zsPMZjPMZvNQohJRlAprz0dEsHbtWlRXV+PYsWPIzc0NGc/Pz0dCQgJqa2v1dc3NzWhpaYHdbgcA2O12NDU1oa2tTZ9TU1MDTdOQl5c3nG0hohgS1p6P0+nEgQMH8PHHHyM1NVU/R2OxWJCUlASLxYKVK1eitLQU6enp0DQN69atg91ux/z58wEACxcuRF5eHpYvX47t27fD7XZjy5YtcDqd3Lsh+ncSzmU0AAMue/fu1efcu3dP3nzzTRk/frwkJyfLK6+8Ijdv3gz5OdeuXZNFixZJUlKSTJgwQd566y3p7e0ddA5eaieKTuE8Nw0iIuqqb2h8Ph8sFgu8Xi80TVMdh4i+Ec5zk+/tIiIlWD5EpATLh4iUYPkQkRIsHyJSguVDREqwfIhICZYPESnB8iEiJYb0rnaikSDBIAJdXQj29MAQF4e4pCQYTCYYDAbV0SgCWD4UcSKC3jt38PXRo/CePo2eW7dgNJuRPHUqMl94AamzZsEQF6c6Jo0ylg9FlIjAf+MGru3cia7mZuCbtxYGOjvhvX0bnU1NyHr9dUwsLGQBjXE850MRFbh7Fy0ffICuK1cgwSD+4ffj7K1b+NLnQ1AEgbt3cf2//gvec+cQg+95pjBwz4ciynvmDDq++AIigpauLmw9fx7NXi/Gxcfj/zz5JF7LzQXu3oWnuhqpM2YgLilJdWQaJdzzoYjqvHQJCAYhAP5vUxMutbcjIAJfby9+c/kyLvzjHwCAu//93wjcu6c2LI0qlg9FTODuXdy7dk2/7evtDRnvCQbhDwT+uYKHXWMay4ciJnDvHu599RUAwADgf9lsiH/gsvqTmoYnUlLu3xCBBIMKUlKk8JwPKWEwGFAydSpSExLw/27exKSkJKx68klkJibqc+TBvSAac1g+pEy80Yj/PWUKXp0yBf37PyEvMGT5jGksH1LKYDBgwNczi3DPZ4zjOR+KWjznM7axfCg6cc9nzGP5UPRi+YxpLB+KWjzsGttYPhSVhIddYx7Lh6IWy2dsY/lQdOKez5jH8qHoxXM+YxrLhyLHYIAxIWFQUyUQgN/jGeVApBLLhyImLjERiZMnD25yMIg+r3d0A5FSLB+KHIOBH41KOpYPRY7BALB86BssH4oYg8EAg5F/cnQf/xIoslg+9A3+JVDk8JwPPYDlQ5HDwy56QFh/Cbt378asWbOgaRo0TYPdbsfRo0f18e7ubjidTmRkZCAlJQXFxcXwPPRajZaWFhQVFSE5ORmZmZnYtGkT+vr6RmZrKLpxz4ceEFb5TJ48Gdu2bUNDQwPOnj2L73//+3j55Zdx8eJFAMDGjRtx6NAhVFVVoa6uDjdu3MCSJUv0+wcCARQVFaGnpwcnT57E/v37sW/fPpSXl4/sVlFUMgA850M6gwzzayHT09OxY8cOvPrqq5g4cSIOHDiAV199FQBw5coVPP3003C5XJg/fz6OHj2KF198ETdu3IDVagUA7NmzBz/5yU/w9ddfw2QyDfgYfr8ffr9fv+3z+ZCdnQ2v1wtN04YTnyJIAgFc27kTdz77bFDzJy1diqylS0c3FI0on88Hi8UyqOfmkP83FAgEUFlZia6uLtjtdjQ0NKC3txcOh0OfM336dOTk5MDlcgEAXC4XZs6cqRcPABQWFsLn8+l7TwOpqKiAxWLRl+zs7KHGJpV42EUPCLt8mpqakJKSArPZjNWrV6O6uhp5eXlwu90wmUxIS0sLmW+1WuF2uwEAbrc7pHj6x/vHvk1ZWRm8Xq++tLa2hhubogFPONMDwv72iqeeegqNjY3wer346KOPUFJSgrq6utHIpjObzTCbzaP6GBQh4ez5iEBEQr9Oh8aMsMvHZDJh6tSpAID8/HycOXMGO3fuxGuvvYaenh60t7eH7P14PB7YbDYAgM1mw+nTp0N+Xv/VsP45RP34Mapj27D3gYPBIPx+P/Lz85GQkIDa2lp9rLm5GS0tLbDb7QAAu92OpqYmtLW16XNqamqgaRry8vKGG4WiXLh7MCyfsS2sPZ+ysjIsWrQIOTk56OjowIEDB/DZZ5/h008/hcViwcqVK1FaWor09HRomoZ169bBbrdj/vz5AICFCxciLy8Py5cvx/bt2+F2u7FlyxY4nU4eVtGjAgFA5P4bUmnMCat82tra8Prrr+PmzZuwWCyYNWsWPv30U/zgBz8AALz77rswGo0oLi6G3+9HYWEh3n//ff3+cXFxOHz4MNasWQO73Y5x48ahpKQE77zzzshuFY0J/BjVsW3Yr/NRIZzXElB0+er993Hrk08GNXdiURGyV63iFbIYEpHX+RCNNgkG7x920ZjE8qHoxcOuMY3lQ1FL+k8405jE8qGoJcEgWD1jF8uHohavdo1tLB+KWjzsGttYPhS9+ArnMY3lQxEVl5w86LmBzk4W0BjG8qGISnriiUF/mmH3//wPgvyI3TGL5UMRxQ8To34sH4oolg/1Y/lQRLF8qB/LhyKK5UP9WD4UWXFx/HweAsDyoQjjng/1Y/lQRLF8qB/LhyKK5UP9WD4UUYa4OPCMDwEsH4o07vnQN1g+FFE87KJ+LB+KKAMvtdM3WD4UUWarFSkzZqiOQVGA5UMRZTSZkPLUU4Oaa87KgjE+7G/0phjB8qGIy3A4YJ406bFzjImJyHzxRRhMpgilokhj+VDEmTIzkf3GG0jIyBhw3GAywbpkCSzz5oX9/e4UO1g+FHEGgwHaM8/gP/7zP6Hl5yMuJQUwGGA0m5GUm4uc1athW7IExoQE1VFpFPHrkkkZEYH09sLvdqOvsxNGkwlmqxVxKSnc44lR4Tw3eTaPlDEYDDCYTEjKyVEdhRTgYRcRKcHyISIlWD5EpATLh4iUYPkQkRIsHyJSguVDREoMq3y2bdsGg8GADRs26Ou6u7vhdDqRkZGBlJQUFBcXw+PxhNyvpaUFRUVFSE5ORmZmJjZt2oQ+fi0u0b+VIZfPmTNn8MEHH2DWrFkh6zdu3IhDhw6hqqoKdXV1uHHjBpYsWaKPBwIBFBUVoaenBydPnsT+/fuxb98+lJeXD30riCj2yBB0dHTItGnTpKamRp577jlZv369iIi0t7dLQkKCVFVV6XMvX74sAMTlcomIyJEjR8RoNIrb7dbn7N69WzRNE7/fP+DjdXd3i9fr1ZfW1lYBIF6vdyjxiWiUeL3eQT83h7Tn43Q6UVRUBIfDEbK+oaEBvb29IeunT5+OnJwcuFwuAIDL5cLMmTNhtVr1OYWFhfD5fLh48eKAj1dRUQGLxaIv2dnZQ4lNRFEk7PKprKzEuXPnUFFR8ciY2+2GyWRCWlpayHqr1Qq3263PebB4+sf7xwZSVlYGr9erL62treHGJqIoE9YbS1tbW7F+/XrU1NQgMTFxtDI9wmw2w2w2R+zxiGj0hbXn09DQgLa2NjzzzDOIj49HfHw86urq8N577yE+Ph5WqxU9PT1ob28PuZ/H44HNZgMA2Gy2R65+9d/un0NEY19Y5bNgwQI0NTWhsbFRX+bMmYNly5bp/05ISEBtba1+n+bmZrS0tMButwMA7HY7mpqa0NbWps+pqamBpmnIy8sboc0iomgX1mFXamoqZjz0zQPjxo1DRkaGvn7lypUoLS1Feno6NE3DunXrYLfbMX/+fADAwoULkZeXh+XLl2P79u1wu93YsmULnE4nD62I/o2M+IeJvfvuuzAajSguLobf70dhYSHef/99fTwuLg6HDx/GmjVrYLfbMW7cOJSUlOCdd94Z6ShEFMX4MapENGLCeW7yvV1EpATLh4iUYPkQkRIsHyJSguVDREqwfIhICZYPESnB8iEiJVg+RKQEy4eIlGD5EJESLB8iUoLlQ0RKsHyISAmWDxEpwfIhIiVYPkSkBMuHiJRg+RCREiwfIlKC5UNESrB8iEgJlg8RKcHyISIlWD5EpATLh4iUYPkQkRIsHyJSguVDREqwfIhICZYPESnB8iEiJVg+RKQEy4eIlGD5EJESLB8iUoLlQ0RKxKsOMBQiAgDw+XyKkxDRg/qfk/3P0ceJyfK5ffs2ACA7O1txEiIaSEdHBywWy2PnxGT5pKenAwBaWlr+5QZGG5/Ph+zsbLS2tkLTNNVxBo25IytWc4sIOjo6kJWV9S/nxmT5GI33T1VZLJaY+g/zIE3TYjI7c0dWLOYe7A4BTzgTkRIsHyJSIibLx2w24+2334bZbFYdJWyxmp25IytWc4fDIIO5JkZENMJics+HiGIfy4eIlGD5EJESLB8iUoLlQ0RKxGT57Nq1C1OmTEFiYiIKCgpw+vRppXlOnDiBl156CVlZWTAYDDh48GDIuIigvLwckyZNQlJSEhwOB7788suQOXfu3MGyZcugaRrS0tKwcuVKdHZ2jmruiooKzJ07F6mpqcjMzMTixYvR3NwcMqe7uxtOpxMZGRlISUlBcXExPB5PyJyWlhYUFRUhOTkZmZmZ2LRpE/r6+kYt9+7duzFr1iz91b92ux1Hjx6N6swD2bZtGwwGAzZs2BBz2UeExJjKykoxmUzyu9/9Ti5evCirVq2StLQ08Xg8yjIdOXJEfvazn8kf/vAHASDV1dUh49u2bROLxSIHDx6Uv/71r/LDH/5QcnNz5d69e/qc559/XmbPni2nTp2Sv/zlLzJ16lRZunTpqOYuLCyUvXv3yoULF6SxsVFeeOEFycnJkc7OTn3O6tWrJTs7W2pra+Xs2bMyf/58+d73vqeP9/X1yYwZM8ThcMj58+flyJEjMmHCBCkrKxu13H/84x/lT3/6k/ztb3+T5uZm+elPfyoJCQly4cKFqM38sNOnT8uUKVNk1qxZsn79en19LGQfKTFXPvPmzROn06nfDgQCkpWVJRUVFQpT/dPD5RMMBsVms8mOHTv0de3t7WI2m+XDDz8UEZFLly4JADlz5ow+5+jRo2IwGOT69esRy97W1iYApK6uTs+ZkJAgVVVV+pzLly8LAHG5XCJyv3iNRqO43W59zu7du0XTNPH7/RHLPn78ePntb38bE5k7Ojpk2rRpUlNTI88995xePrGQfSTF1GFXT08PGhoa4HA49HVGoxEOhwMul0thsm939epVuN3ukMwWiwUFBQV6ZpfLhbS0NMyZM0ef43A4YDQaUV9fH7GsXq8XwD8/NaChoQG9vb0h2adPn46cnJyQ7DNnzoTVatXnFBYWwufz4eLFi6OeORAIoLKyEl1dXbDb7TGR2el0oqioKCQjEBu/75EUU+9qv3XrFgKBQMgvHgCsViuuXLmiKNXjud1uABgwc/+Y2+1GZmZmyHh8fDzS09P1OaMtGAxiw4YNePbZZzFjxgw9l8lkQlpa2mOzD7Rt/WOjpampCXa7Hd3d3UhJSUF1dTXy8vLQ2NgYtZkBoLKyEufOncOZM2ceGYvm3/doiKnyodHjdDpx4cIFfP7556qjDMpTTz2FxsZGeL1efPTRRygpKUFdXZ3qWI/V2tqK9evXo6amBomJiarjKBdTh10TJkxAXFzcI2f/PR4PbDabolSP15/rcZltNhva2tpCxvv6+nDnzp2IbNfatWtx+PBhHD9+HJMnT9bX22w29PT0oL29/bHZB9q2/rHRYjKZMHXqVOTn56OiogKzZ8/Gzp07ozpzQ0MD2tra8MwzzyA+Ph7x8fGoq6vDe++9h/j4eFit1qjNPhpiqnxMJhPy8/NRW1urrwsGg6itrYXdbleY7Nvl5ubCZrOFZPb5fKivr9cz2+12tLe3o6GhQZ9z7NgxBINBFBQUjFo2EcHatWtRXV2NY8eOITc3N2Q8Pz8fCQkJIdmbm5vR0tISkr2pqSmkPGtqaqBpGvLy8kYt+8OCwSD8fn9UZ16wYAGamprQ2NioL3PmzMGyZcv0f0dr9lGh+ox3uCorK8VsNsu+ffvk0qVL8sYbb0haWlrI2f9I6+jokPPnz8v58+cFgPzyl7+U8+fPy1dffSUi9y+1p6WlyccffyxffPGFvPzyywNeav/ud78r9fX18vnnn8u0adNG/VL7mjVrxGKxyGeffSY3b97Ul7t37+pzVq9eLTk5OXLs2DE5e/as2O12sdvt+nj/pd+FCxdKY2OjfPLJJzJx4sRRvfS7efNmqaurk6tXr8oXX3whmzdvFoPBIH/+85+jNvO3efBqV6xlH66YKx8RkV//+teSk5MjJpNJ5s2bJ6dOnVKa5/jx4wLgkaWkpERE7l9u37p1q1itVjGbzbJgwQJpbm4O+Rm3b9+WpUuXSkpKimiaJitWrJCOjo5RzT1QZgCyd+9efc69e/fkzTfflPHjx0tycrK88sorcvPmzZCfc+3aNVm0aJEkJSXJhAkT5K233pLe3t5Ry/3jH/9YnnjiCTGZTDJx4kRZsGCBXjzRmvnbPFw+sZR9uPh5PkSkREyd8yGisYPlQ0RKsHyISAmWDxEpwfIhIiVYPkSkBMuHiJRg+RCREiwfIlKC5UNESrB8iEiJ/w+C1zR3c2cjaQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "#打印游戏\n",
    "def show():\n",
    "    plt.figure(figsize=(3, 3))\n",
    "    plt.imshow(env.render())\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f489de60",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env.observation_space= Box([-1. -1. -8.], [1. 1. 8.], (3,), float32)\n",
      "env.action_space= Box(-2.0, 2.0, (1,), float32)\n",
      "state= [ 0.9314149  -0.3639592   0.71713585]\n",
      "action= [-1.0667996]\n",
      "next_state= [ 0.9364916  -0.35068998  0.2841465 ]\n",
      "reward= -0.19133394317625788\n",
      "done= False\n",
      "info= {}\n"
     ]
    }
   ],
   "source": [
    "#认识游戏环境\n",
    "def test_env():\n",
    "    print('env.observation_space=', env.observation_space)\n",
    "    print('env.action_space=', env.action_space)\n",
    "\n",
    "    state = env.reset()\n",
    "    action = env.action_space.sample()\n",
    "    next_state, reward, done, info = env.step(action)\n",
    "\n",
    "    print('state=', state)\n",
    "    print('action=', action)\n",
    "    print('next_state=', next_state)\n",
    "    print('reward=', reward)\n",
    "    print('done=', done)\n",
    "    print('info=', info)\n",
    "\n",
    "\n",
    "test_env()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6d5c572d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<stable_baselines3.a2c.a2c.A2C at 0x7fd8f0ac7a90>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stable_baselines3 import A2C\n",
    "from stable_baselines3.common.env_util import make_vec_env\n",
    "\n",
    "#初始化模型\n",
    "model = A2C(\n",
    "    policy='MlpPolicy',\n",
    "    env=make_vec_env(MyWrapper, n_envs=8),  #使用N个环境同时训练\n",
    "    learning_rate=1e-3,\n",
    "    n_steps=5,  #运行N步后执行更新,batch_size=n_steps*环境数量\n",
    "    gamma=0.9,\n",
    "    verbose=0)\n",
    "\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "22afd2bc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/stable_baselines3/common/evaluation.py:67: UserWarning: Evaluation environment is not wrapped with a ``Monitor`` wrapper. This may result in reporting modified episode lengths and rewards, if other wrappers happen to modify these. Consider wrapping environment first with ``Monitor`` wrapper.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(-1247.169250970462, 287.27973636828875)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stable_baselines3.common.evaluation import evaluate_policy\n",
    "\n",
    "evaluate_policy(model, env, n_eval_episodes=20, deterministic=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "75e0b593",
   "metadata": {},
   "outputs": [],
   "source": [
    "# #训练\n",
    "# model.learn(200_0000, progress_bar=True)\n",
    "\n",
    "# #保存模型\n",
    "# model.save('save/3.A2C.Pendulum')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b05f4766",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-319.25174775233245, 237.00185868279056)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#加载模型\n",
    "model = A2C.load('save/3.A2C.Pendulum')\n",
    "\n",
    "evaluate_policy(model, env, n_eval_episodes=20, deterministic=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "414e1478",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAR8AAAEXCAYAAACUBEAgAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAa4klEQVR4nO3df2xT973/8ZedxA4kOQ4BYpOSlExwCxE/VgIEr1I7DY+0zboyMt2OL2IZQ61gBvFjQiMbpBqaFC6V1pUNwq6qAfpe0eymt6EjhXZRoGEVJkAga/iVdfeyJhewQ2GxAyN2Yr/vH23OMAQah8QfO3k9JEvknI/jtyl+9tgndgwiIiAiijKj6gGIaGRifIhICcaHiJRgfIhICcaHiJRgfIhICcaHiJRgfIhICcaHiJRgfIhICWXx2blzJyZNmoTk5GQUFBTg5MmTqkYhIgWUxOf3v/89NmzYgFdffRVnzpzBrFmzUFhYiPb2dhXjEJECBhVvLC0oKMDcuXPxm9/8BgAQCoWQnZ2NNWvWYNOmTV96/VAohKtXryItLQ0Gg2GoxyWifhIRdHZ2IisrC0bjw49tEqM0ky4QCKCxsRGlpaX6NqPRCIfDAZfL1ed1/H4//H6//vWVK1eQl5c35LMS0cC0tbVh4sSJD10T9fh89tlnCAaDsFqtYdutVisuXbrU53XKy8vx85///L7tbW1t0DRtSOYkosj5fD5kZ2cjLS3tS9dGPT4DUVpaig0bNuhf995BTdMYH6IY1J+XQ6Ien3HjxiEhIQEejydsu8fjgc1m6/M6ZrMZZrM5GuMRUZRE/WyXyWRCfn4+6urq9G2hUAh1dXWw2+3RHoeIFFHytGvDhg0oKSnBnDlzMG/ePPzqV7/C7du3sXz5chXjEJECSuLz0ksv4fr16ygrK4Pb7cZXv/pVvP/++/e9CE1Ew5eSn/N5VD6fDxaLBV6vly84E8WQSB6bfG8XESnB+BCREowPESnB+BCREowPESnB+BCREowPESnB+BCREowPESnB+BCREowPESnB+BCREowPESnB+BCREowPESnB+BCREowPESnB+BCREowPESnB+BCREowPESnB+BCREowPESnB+BCREowPESnB+BCREowPESnB+BCREowPESnB+BCREowPESnB+BCREowPESnB+BCREowPESkRcXyOHTuGF154AVlZWTAYDDhw4EDYfhFBWVkZJkyYgFGjRsHhcOCTTz4JW3Pz5k0sXboUmqYhPT0dK1aswK1btx7pjhBRfIk4Prdv38asWbOwc+fOPvdv374dO3bswO7du9HQ0ICUlBQUFhaiq6tLX7N06VKcP38etbW1qKmpwbFjx/DKK68M/F4QUfyRRwBAqqur9a9DoZDYbDZ57bXX9G0dHR1iNpvlrbfeEhGRCxcuCAA5deqUvubw4cNiMBjkypUr/bpdr9crAMTr9T7K+EQ0yCJ5bA7qaz6XL1+G2+2Gw+HQt1ksFhQUFMDlcgEAXC4X0tPTMWfOHH2Nw+GA0WhEQ0NDn9/X7/fD5/OFXYgovg1qfNxuNwDAarWGbbdarfo+t9uNzMzMsP2JiYnIyMjQ19yrvLwcFotFv2RnZw/m2ESkQFyc7SotLYXX69UvbW1tqkciokc0qPGx2WwAAI/HE7bd4/Ho+2w2G9rb28P29/T04ObNm/qae5nNZmiaFnYhovg2qPHJzc2FzWZDXV2dvs3n86GhoQF2ux0AYLfb0dHRgcbGRn3NkSNHEAqFUFBQMJjjEFEMS4z0Crdu3cJf//pX/evLly+jqakJGRkZyMnJwbp16/CLX/wCU6ZMQW5uLrZs2YKsrCwsWrQIADBt2jQ8++yzePnll7F79250d3dj9erV+N73voesrKxBu2NEFOMiPZV29OhRAXDfpaSkREQ+P92+ZcsWsVqtYjabZcGCBdLS0hL2PW7cuCFLliyR1NRU0TRNli9fLp2dnf2egafaiWJTJI9Ng4iIwvYNiM/ng8Vigdfr5es/RDEkksdmXJztIqLhh/EhIiUYHyJSIuKzXUSPSkQQ7OyE3+1GyO9HQloakidMgMFkgsFgUD0eRQnjQ1EVCgRw4+hRXH/vPXRduwYJBJCQkoKUf/kXTPjXf0XKtGkM0AjB+FDUhAIBXKuqgueddyDd3fr24K1b8J05gzufforHV6+GNns2AzQC8DUfigoRQUdDA9oPHAgLz926b9xA27//OwL3vP2GhifGh6Ii5Pej/eBBhPz+h67zX7uGG3e9PYeGL8aHokJ6euB/wEem3Kvrf/93iKehWMD4UMwREcThD95ThBgfij3BIBAKqZ6ChhjjQzFHgkEe+YwAjA/FHOGRz4jA+FDMkZ4egEc+wx7jQzFHgkEIj3yGPcaHYo4EgzzyGQEYH4o50tPDI58RgPGhmNPj8yEUCKgeg4YY40Mxp/vmTYTu3FE9Bg0xxoeIlGB8iEgJxoeIlGB8KCqMSUlInjhR9RgUQxgfig6DAQkpKaqnoBjC+FB0GAwwGPnPjf6J/xooagwJCapHoBjC+FB0GAyRxYdvrxj2GB+KCgMARPC0S3p6hmwWig2MD0VHhEc+IcZn2GN8KGoiic+Dfr0ODR+MD0VNv892ifBp1wjA+FB0GAxAP498BHzaNRIwPhQ1fNpFd2N8KDoi+SFDPu0aERgfipqIznbxyGfYiyg+5eXlmDt3LtLS0pCZmYlFixahpaUlbE1XVxecTifGjh2L1NRUFBcXw+PxhK1pbW1FUVERRo8ejczMTGzcuBE9/D/d8NffI59QCHf+53+GdhZSLqL41NfXw+l04sSJE6itrUV3dzcWLlyI27dv62vWr1+PgwcPoqqqCvX19bh69SoWL16s7w8GgygqKkIgEMDx48exb98+7N27F2VlZYN3ryjmGAyGiNaH/P4hmoRihUEe4VdDXr9+HZmZmaivr8fTTz8Nr9eL8ePHY//+/fjud78LALh06RKmTZsGl8uF+fPn4/Dhw/jWt76Fq1evwmq1AgB2796Nn/zkJ7h+/TpMJtOX3q7P54PFYoHX64WmaQMdn6Lsyn/8B9z/+Z/9Wjv++eeRs3LlEE9Egy2Sx+Yjvebj9XoBABkZGQCAxsZGdHd3w+Fw6GumTp2KnJwcuFwuAIDL5cKMGTP08ABAYWEhfD4fzp8/3+ft+P1++Hy+sAsRxbcBxycUCmHdunV46qmnMH36dACA2+2GyWRCenp62Fqr1Qq3262vuTs8vft79/WlvLwcFotFv2RnZw90bCKKEQOOj9PpxLlz51BZWTmY8/SptLQUXq9Xv7S1tQ35bRLR0EocyJVWr16NmpoaHDt2DBPv+mhMm82GQCCAjo6OsKMfj8cDm82mrzl58mTY9+s9G9a75l5msxlms3kgoxJRjIroyEdEsHr1alRXV+PIkSPIzc0N25+fn4+kpCTU1dXp21paWtDa2gq73Q4AsNvtaG5uRnt7u76mtrYWmqYhLy/vUe4LEcWRiI58nE4n9u/fj3fffRdpaWn6azQWiwWjRo2CxWLBihUrsGHDBmRkZEDTNKxZswZ2ux3z588HACxcuBB5eXlYtmwZtm/fDrfbjc2bN8PpdPLohsKISMSn6Cl+RBSfiooKAMDXv/71sO179uzBD37wAwDA66+/DqPRiOLiYvj9fhQWFmLXrl362oSEBNTU1GDVqlWw2+1ISUlBSUkJtm7d+mj3hIYV/q724S+i+PTnR4KSk5Oxc+dO7Ny584FrHn/8cRw6dCiSm6YRRnp6Pv8oVR75DFt8bxfFJD0+NGwxPhSTpKenX0faFL8YH4qapDFj+v3mUgkGeeQzzDE+FDWmzMx+f6wGn3YNf4wPRU1En2TI+Ax7jA9FTUTxCQb5ms8wx/hQ1BiMxn6fOu+5dYsfpTrMMT4UNZEc+fjdboQCgSGchlRjfCh6EhL4Q4OkY3woagwJCWB6qBfjQ1HT71+dQyMC/zVQ1Bj4tIvuMqAPEyMaiHtfcBYRdAQC+GtnJ0YnJGCKxYIkg4EfozFCMD4UPXfFR0TQevs2tpw9ixavF2ajES/l5uLlJ56AifEZEfi0i6Lm7p/zEQD/1tyMCx0dCIrgH8Eg/v9//zcu/P3vaoekqGF8KGoMSUkw3vVplb57fiVyjwgC/BCxEYPxoahJSk9HxtNPAwAMAL5usyHxrqdYj6ek4LHRowEACWYzz44Nc3zNh6LGYDRinMOBG0eOIHjrFkomT0ZyQgKOXruGDLMZy6dMQdYX8RnzzDNI5G+jHdYYH4qq5JwcTFiyBFf27UOi34+lX/kK/t9XvqL/8KHBYEBqXh6sixbxyGeYY3woqgxGI8Y/+ywAwPNf/4Xuv/8dhi/evW5ISoL25JOYuGIFTF/8Cm4avhgfijpjUhIyi4pgmT0bvrNn4Xe7YRw1CqnTpiF12jQkjBqlekSKAsaHlDAYjUh+7DEkP/aY6lFIET6pJiIlGB8iUoLxISIlGB8iUoLxISIlGB8iUoLxISIlGB8iUoLxISIlGB8iUoLxISIlGB8iUoLxISIlGB8iUiKi+FRUVGDmzJnQNA2apsFut+Pw4cP6/q6uLjidTowdOxapqakoLi6Gx+MJ+x6tra0oKirC6NGjkZmZiY0bN6Knp2dw7g0RxY2I4jNx4kRs27YNjY2NOH36NL7xjW/gxRdfxPnz5wEA69evx8GDB1FVVYX6+npcvXoVixcv1q8fDAZRVFSEQCCA48ePY9++fdi7dy/KysoG914RUeyTRzRmzBh58803paOjQ5KSkqSqqkrfd/HiRQEgLpdLREQOHTokRqNR3G63vqaiokI0TRO/3//A2+jq6hKv16tf2traBIB4vd5HHZ+IBpHX6+33Y3PAr/kEg0FUVlbi9u3bsNvtaGxsRHd3NxwOh75m6tSpyMnJgcvlAgC4XC7MmDEDVqtVX1NYWAifz6cfPfWlvLwcFotFv2RnZw90bCKKERHHp7m5GampqTCbzVi5ciWqq6uRl5cHt9sNk8mE9PT0sPVWqxVutxsA4Ha7w8LTu79334OUlpbC6/Xql7a2tkjHJqIYE/FnOD/xxBNoamqC1+vF22+/jZKSEtTX1w/FbDqz2QzzXb/pkojiX8TxMZlMmDx5MgAgPz8fp06dwhtvvIGXXnoJgUAAHR0dYUc/Ho8HNpsNAGCz2XDy5Mmw79d7Nqx3DRGNDI/8cz6hUAh+vx/5+flISkpCXV2dvq+lpQWtra2w2+0AALvdjubmZrS3t+tramtroWka8vLyHnUUIoojER35lJaW4rnnnkNOTg46Ozuxf/9+fPjhh/jggw9gsViwYsUKbNiwARkZGdA0DWvWrIHdbsf8+fMBAAsXLkReXh6WLVuG7du3w+12Y/PmzXA6nXxaRTTCRBSf9vZ2fP/738e1a9dgsVgwc+ZMfPDBB/jmN78JAHj99ddhNBpRXFwMv9+PwsJC7Nq1S79+QkICampqsGrVKtjtdqSkpKCkpARbt24d3HtFRDHPIPLF76qNIz6fDxaLBV6vF5qmqR6HiL4QyWOT7+0iIiUYHyJSgvEhIiUYHyJSgvEhIiUYHyJSgvEhIiUYHyJSgvEhIiUYHyJSgvEhIiUYHyJSgvEhIiUYHyJSgvEhIiUYHyJSgvEhIiUYHyJSgvEhIiUYHyJSgvEhIiUYHyJSgvEhIiUYHyJSgvEhIiUYHyJSgvEhIiUYHyJSgvEhIiUYHyJSgvEhIiUYHyJSgvEhIiUYHyJSgvEhIiUeKT7btm2DwWDAunXr9G1dXV1wOp0YO3YsUlNTUVxcDI/HE3a91tZWFBUVYfTo0cjMzMTGjRvR09PzKKMQUZwZcHxOnTqF3/72t5g5c2bY9vXr1+PgwYOoqqpCfX09rl69isWLF+v7g8EgioqKEAgEcPz4cezbtw979+5FWVnZwO8FEcUfGYDOzk6ZMmWK1NbWyjPPPCNr164VEZGOjg5JSkqSqqoqfe3FixcFgLhcLhEROXTokBiNRnG73fqaiooK0TRN/H5/n7fX1dUlXq9Xv7S1tQkA8Xq9AxmfiIaI1+vt92NzQEc+TqcTRUVFcDgcYdsbGxvR3d0dtn3q1KnIycmBy+UCALhcLsyYMQNWq1VfU1hYCJ/Ph/Pnz/d5e+Xl5bBYLPolOzt7IGMTUQyJOD6VlZU4c+YMysvL79vndrthMpmQnp4ett1qtcLtdutr7g5P7/7efX0pLS2F1+vVL21tbZGOTUQxJjGSxW1tbVi7di1qa2uRnJw8VDPdx2w2w2w2R+32iGjoRXTk09jYiPb2dsyePRuJiYlITExEfX09duzYgcTERFitVgQCAXR0dIRdz+PxwGazAQBsNtt9Z796v+5dQ0TDX0TxWbBgAZqbm9HU1KRf5syZg6VLl+p/TkpKQl1dnX6dlpYWtLa2wm63AwDsdjuam5vR3t6ur6mtrYWmacjLyxuku0VEsS6ip11paWmYPn162LaUlBSMHTtW375ixQps2LABGRkZ0DQNa9asgd1ux/z58wEACxcuRF5eHpYtW4bt27fD7XZj8+bNcDqdfGpFNIJEFJ/+eP3112E0GlFcXAy/34/CwkLs2rVL35+QkICamhqsWrUKdrsdKSkpKCkpwdatWwd7FCKKYQYREdVDRMrn88FiscDr9ULTNNXjENEXInls8r1dRKQE40NESjA+RKQE40NESjA+RKQE40NESjA+RKQE40NESjA+RKQE40NESjA+RKQE40NESjA+RKQE40NESjA+RKQE40NESjA+RKQE40NESjA+RKQE40NESjA+RKQE40NESjA+RKQE40NESjA+RKQE40NESjA+RKQE40NESjA+RKQE40NESjA+RKQE40NESjA+RKQE40NESjA+RKQE40NESjA+RKREouoBBkJEAAA+n0/xJER0t97HZO9j9GHiMj43btwAAGRnZyuehIj60tnZCYvF8tA1cRmfjIwMAEBra+uX3sFY4/P5kJ2djba2NmiapnqcfuPc0RWvc4sIOjs7kZWV9aVr4zI+RuPnL1VZLJa4+g9zN03T4nJ2zh1d8Th3fw8I+IIzESnB+BCREnEZH7PZjFdffRVms1n1KBGL19k5d3TF69yRMEh/zokREQ2yuDzyIaL4x/gQkRKMDxEpwfgQkRKMDxEpEZfx2blzJyZNmoTk5GQUFBTg5MmTSuc5duwYXnjhBWRlZcFgMODAgQNh+0UEZWVlmDBhAkaNGgWHw4FPPvkkbM3NmzexdOlSaJqG9PR0rFixArdu3RrSucvLyzF37lykpaUhMzMTixYtQktLS9iarq4uOJ1OjB07FqmpqSguLobH4wlb09raiqKiIowePRqZmZnYuHEjenp6hmzuiooKzJw5U//pX7vdjsOHD8f0zH3Ztm0bDAYD1q1bF3ezDwqJM5WVlWIymeR3v/udnD9/Xl5++WVJT08Xj8ejbKZDhw7Jz372M3nnnXcEgFRXV4ft37Ztm1gsFjlw4ID8+c9/lm9/+9uSm5srd+7c0dc8++yzMmvWLDlx4oT86U9/ksmTJ8uSJUuGdO7CwkLZs2ePnDt3TpqamuT555+XnJwcuXXrlr5m5cqVkp2dLXV1dXL69GmZP3++fO1rX9P39/T0yPTp08XhcMjZs2fl0KFDMm7cOCktLR2yuf/whz/Ie++9J3/5y1+kpaVFfvrTn0pSUpKcO3cuZme+18mTJ2XSpEkyc+ZMWbt2rb49HmYfLHEXn3nz5onT6dS/DgaDkpWVJeXl5Qqn+qd74xMKhcRms8lrr72mb+vo6BCz2SxvvfWWiIhcuHBBAMipU6f0NYcPHxaDwSBXrlyJ2uzt7e0CQOrr6/U5k5KSpKqqSl9z8eJFASAul0tEPg+v0WgUt9utr6moqBBN08Tv90dt9jFjxsibb74ZFzN3dnbKlClTpLa2Vp555hk9PvEw+2CKq6ddgUAAjY2NcDgc+jaj0QiHwwGXy6Vwsge7fPky3G532MwWiwUFBQX6zC6XC+np6ZgzZ46+xuFwwGg0oqGhIWqzer1eAP/81IDGxkZ0d3eHzT516lTk5OSEzT5jxgxYrVZ9TWFhIXw+H86fPz/kMweDQVRWVuL27duw2+1xMbPT6URRUVHYjEB8/H0Pprh6V/tnn32GYDAY9hcPAFarFZcuXVI01cO53W4A6HPm3n1utxuZmZlh+xMTE5GRkaGvGWqhUAjr1q3DU089henTp+tzmUwmpKenP3T2vu5b776h0tzcDLvdjq6uLqSmpqK6uhp5eXloamqK2ZkBoLKyEmfOnMGpU6fu2xfLf99DIa7iQ0PH6XTi3Llz+Oijj1SP0i9PPPEEmpqa4PV68fbbb6OkpAT19fWqx3qotrY2rF27FrW1tUhOTlY9jnJx9bRr3LhxSEhIuO/Vf4/HA5vNpmiqh+ud62Ez22w2tLe3h+3v6enBzZs3o3K/Vq9ejZqaGhw9ehQTJ07Ut9tsNgQCAXR0dDx09r7uW+++oWIymTB58mTk5+ejvLwcs2bNwhtvvBHTMzc2NqK9vR2zZ89GYmIiEhMTUV9fjx07diAxMRFWqzVmZx8KcRUfk8mE/Px81NXV6dtCoRDq6upgt9sVTvZgubm5sNlsYTP7fD40NDToM9vtdnR0dKCxsVFfc+TIEYRCIRQUFAzZbCKC1atXo7q6GkeOHEFubm7Y/vz8fCQlJYXN3tLSgtbW1rDZm5ubw+JZW1sLTdOQl5c3ZLPfKxQKwe/3x/TMCxYsQHNzM5qamvTLnDlzsHTpUv3PsTr7kFD9inekKisrxWw2y969e+XChQvyyiuvSHp6etir/9HW2dkpZ8+elbNnzwoA+eUvfylnz56VTz/9VEQ+P9Wenp4u7777rnz88cfy4osv9nmq/cknn5SGhgb56KOPZMqUKUN+qn3VqlVisVjkww8/lGvXrumXf/zjH/qalStXSk5Ojhw5ckROnz4tdrtd7Ha7vr/31O/ChQulqalJ3n//fRk/fvyQnvrdtGmT1NfXy+XLl+Xjjz+WTZs2icFgkD/+8Y8xO/OD3H22K95mf1RxFx8RkV//+teSk5MjJpNJ5s2bJydOnFA6z9GjRwXAfZeSkhIR+fx0+5YtW8RqtYrZbJYFCxZIS0tL2Pe4ceOGLFmyRFJTU0XTNFm+fLl0dnYO6dx9zQxA9uzZo6+5c+eO/OhHP5IxY8bI6NGj5Tvf+Y5cu3Yt7Pv87W9/k+eee05GjRol48aNkx//+MfS3d09ZHP/8Ic/lMcff1xMJpOMHz9eFixYoIcnVmd+kHvjE0+zPyp+ng8RKRFXr/kQ0fDB+BCREowPESnB+BCREowPESnB+BCREowPESnB+BCREowPESnB+BCREowPESnxfz32/aewQVeoAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-552.9787354795611 200 [-6.403364529886047, -6.559451885950866, -6.748173157209415, -6.963253276383593, -7.197324400447656, -7.4421762566494225, -7.685702878753752, -8.011315249067252, -8.459531587420715, -9.098510053384656, -9.925879078787965, -10.149264029395011, -9.445930569893216, -8.689868121075914, -7.904861475428319, -7.117850854349222, -6.356961530227772, -5.648779500529242, -5.015805630954743, -4.474912934108402, -4.037135342960264, -3.708585894872944, -3.4920001320475516, -3.3883667575060783, -3.3982339678299924, -3.522440641779429, -3.762148023874511, -4.115998200501329, -4.639151657699833, -5.4589390140439065, -6.6816527282607066, -8.287440847676086, -10.209750584506137, -12.33837108003363, -11.785390804544472, -10.322449551850136, -8.805717924324806, -7.1012635254092915, -5.389966009745247, -4.266574138158974, -3.4008778125979338, -2.6928045577995054, -2.1295936927604435, -1.6917974921198649, -1.3581702708800174, -1.1087792548181659, -0.9265868823311403, -0.7980010230152399, -0.7128536762365798, -0.6641396038902014, -0.6477205364824076, -0.662109087973261, -0.7083888961657474, -0.7902925866671782, -0.9144336623506819, -1.0906589569141147, -1.3324408225830904, -1.6571481016393907, -2.085911082317724, -2.642634295074285, -3.351565255568252, -4.232840476727187, -5.292723443695252, -6.707693066662953, -8.256759335347166, -9.78448281885315, -11.300400123255537, -12.262561470865409, -10.632979205165002, -9.342451692801092, -8.025659169073716, -6.741241239992302, -5.546370747534272, -4.484445233067842, -3.5786965943047804, -2.8328340184437493, -2.236183153850967, -1.7700089737166753, -1.412827086888545, -1.143929748993989, -0.9452657655540135, -0.8021583535109563, -0.7033492667412954, -0.640734052054951, -0.6090197932604935, -0.605435764018409, -0.6295633818057137, -0.6833152415529405, -0.7710716471480528, -0.8999653007294907, -1.0802791099617999, -1.3258757018861729, -1.6544962409801687, -2.0876397941033837, -2.6495695104534325, -3.364843084391339, -4.2537800020214265, -5.32572028877216, -6.567075301763922, -8.314477013651018, -10.664910318079897, -13.28737053135712, -13.29305088014477, -11.553682094456112, -9.564742829354806, -7.220396099622464, -5.259723496851284, -4.012779583373551, -3.125618920979947, -2.415043087535065, -1.8593990873247748, -1.4330380094179997, -1.1106891631634652, -0.8704834926403211, -0.6953126062663473, -0.5701063780449889, -0.43740088757308965, -0.2814406958600056, -0.17301751710489868, -0.09424878476279315, -0.052793639723829396, -0.02848832992747617, -0.019349019443678662, -0.013641034765059843, -0.00889911077293328, -0.005378319616842694, -0.0038553754861721527, -0.003030665172404705, -0.002297663157483803, -0.0017788564655346418, -0.001502406380101694, -0.0012849365687527116, -0.0009765607553700701, -0.000902358492032804, -0.0007777253114863694, -0.0006693808578542544, -0.0007023542446344618, -0.0006469077370246669, -0.000617331647270525, -0.0005807202343211575, -0.0007112612322553668, -0.0005698962610725659, -0.000605942261589215, -0.0005453168056284277, -0.0005021330589351212, -0.0004839684888703656, -0.0004796299793731146, -0.0004496601152976743, -0.00042959433022526314, -0.00041846047563522077, -0.00040842441128936894, -0.0005392911990528344, -0.00043502517589950277, -0.0005619842033430516, -0.00046294484522016303, -0.00046284426966360125, -0.0006491328379841488, -0.0005347761156479717, -0.0006079417289124552, -0.00052303300736512, -0.000575562388018287, -0.0005702801795887955, -0.0006382720189492246, -0.0005800461438617601, -0.000652896601372401, -0.0005894599365288421, -0.0004888018605945698, -0.0006479462971399484, -0.0005235105151487691, -0.0006093923260457232, -0.0005087482036524032, -0.0005818800949996132, -0.0006035512694374148, -0.0005516848311199498, -0.0005075725568097035, -0.0004865052465299227, -0.0005299420132831919, -0.0005313521159214184, -0.00048159235351189964, -0.00046782832720119533, -0.0004733587608534474, -0.0005288968176221472, -0.0005715594870956539, -0.0005621842795695428, -0.0005812726888651275, -0.0005238569283054104, -0.0005017928572677193, -0.00048779569477528024, -0.0005796981739451, -0.0005513252729213517, -0.0005194369867877167, -0.0005218780541503228, -0.0004859298507840392, -0.0005432498823225844, -0.000533616751217941, -0.0005199808604596199, -0.0005052143066023264, -0.0004950109796168234, -0.0004121028623572278, -0.0005784799940535202]\n"
     ]
    }
   ],
   "source": [
    "from IPython import display\n",
    "import random\n",
    "\n",
    "\n",
    "def test():\n",
    "    state = env.reset()\n",
    "    reward_sum = []\n",
    "    over = False\n",
    "    while not over:\n",
    "        action, _ = model.predict(state)\n",
    "        state, reward, over, _ = env.step(action)\n",
    "        reward_sum.append(reward)\n",
    "\n",
    "        if len(reward_sum) % 5 == 0:\n",
    "            display.clear_output(wait=True)\n",
    "            show()\n",
    "\n",
    "    print(sum(reward_sum), len(reward_sum), reward_sum)\n",
    "\n",
    "\n",
    "test()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
